From ed94e7ef5496ca54f5ddd3c5035dc29c7812daaa Mon Sep 17 00:00:00 2001 From: Adrian Vondendriesch Date: Tue, 26 Mar 2013 09:33:34 +0100 Subject: [PATCH] Add APT repository webpages at /repos/apt/ The repository contents are imported into a (mostly) normalized database schema (apt_* tables), from where the web pages are served. Dependencies: python-apt, postgresql-debversion, pg_trgm Authors: Adrian Vondendriesch and Christoph Berg --- pgweb/apt/__init__.py | 0 pgweb/apt/models.py | 120 ++++++++ pgweb/apt/sql/architecture.sql | 1 + pgweb/apt/sql/package.sql | 2 + pgweb/apt/sql/packagecontents.sql | 2 + pgweb/apt/sql/pkg.sql | 12 + pgweb/apt/sql/release.sql | 2 + pgweb/apt/sql/source.sql | 2 + pgweb/apt/sql/srcsuite.sql | 2 + pgweb/apt/sql/suite.sql | 2 + pgweb/apt/sql/vendor.sql | 1 + pgweb/apt/utils.py | 29 ++ pgweb/apt/views.py | 373 ++++++++++++++++++++++++ pgweb/settings.py | 3 + pgweb/urls.py | 32 +++ pgweb/util/contexts.py | 9 + templates/apt/apt.html | 80 ++++++ templates/apt/binaries.html | 46 +++ templates/apt/binary.html | 167 +++++++++++ templates/apt/binary_contents.html | 14 + templates/apt/dependency.html | 12 + templates/apt/distributions.html | 20 ++ templates/apt/madison.html | 50 ++++ templates/apt/qa.html | 33 +++ templates/apt/search.html | 87 ++++++ templates/apt/source.html | 136 +++++++++ templates/apt/source_file.html | 66 +++++ templates/apt/sources.html | 18 ++ tools/apt/import-packagelist.py | 440 +++++++++++++++++++++++++++++ 29 files changed, 1761 insertions(+) create mode 100644 pgweb/apt/__init__.py create mode 100644 pgweb/apt/models.py create mode 100644 pgweb/apt/sql/architecture.sql create mode 100644 pgweb/apt/sql/package.sql create mode 100644 pgweb/apt/sql/packagecontents.sql create mode 100644 pgweb/apt/sql/pkg.sql create mode 100644 pgweb/apt/sql/release.sql create mode 100644 pgweb/apt/sql/source.sql create mode 100644 pgweb/apt/sql/srcsuite.sql create mode 100644 pgweb/apt/sql/suite.sql create mode 100644 pgweb/apt/sql/vendor.sql create mode 100644 pgweb/apt/utils.py create mode 100644 pgweb/apt/views.py create mode 100644 templates/apt/apt.html create mode 100644 templates/apt/binaries.html create mode 100644 templates/apt/binary.html create mode 100644 templates/apt/binary_contents.html create mode 100644 templates/apt/dependency.html create mode 100644 templates/apt/distributions.html create mode 100644 templates/apt/madison.html create mode 100644 templates/apt/qa.html create mode 100644 templates/apt/search.html create mode 100644 templates/apt/source.html create mode 100644 templates/apt/source_file.html create mode 100644 templates/apt/sources.html create mode 100755 tools/apt/import-packagelist.py diff --git a/pgweb/apt/__init__.py b/pgweb/apt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pgweb/apt/models.py b/pgweb/apt/models.py new file mode 100644 index 0000000..fbc5fa9 --- /dev/null +++ b/pgweb/apt/models.py @@ -0,0 +1,120 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch +# Christoph Berg + +from django.db import models +from datetime import datetime + +class Vendor(models.Model): + vendor = models.TextField(null=False, primary_key=True) + + def __unicode(self): + return vendor; + +class Release(models.Model): + vendor = models.ForeignKey(Vendor) + release = models.TextField(null=False, primary_key=True) + relversion = models.TextField() + active = models.BooleanField(null=False, default=True) + + def __unicode(self): + return release; + +class Architecture(models.Model): + architecture = models.TextField(null=False, primary_key=True) + + def __unicode(self): + return architecture; + +class Component(models.Model): + component = models.TextField(null=False, primary_key=True) + + def __unicode__(self): + return self.component + +class Suite(models.Model): + release = models.ForeignKey(Release) + component = models.ForeignKey(Component) + architecture = models.ForeignKey(Architecture) + last_update = models.DateTimeField(null=True) + active = models.BooleanField(null=False, default=True) + + class Meta: + unique_together = (("release", "component", "architecture"),) + +class Srcsuite(models.Model): + release = models.ForeignKey(Release) + component = models.ForeignKey(Component) + last_update = models.DateTimeField(null=True) + active = models.BooleanField(null=False, default=True) + + class Meta: + unique_together = (("release", "component"),) + +class Package(models.Model): + package = models.TextField(null=False) + version = models.TextField(null=False) + arch = models.ForeignKey(Architecture) + source = models.TextField(null=False) + srcversion = models.TextField(null=False) + maintainer = models.TextField(null=False) + short_description = models.TextField(null=False) + description = models.TextField(null=True) + homepage = models.TextField(null=True) + installed_size = models.IntegerField(null=False) + filename = models.TextField(null=False) + depends = models.TextField(null=True) + recommends = models.TextField(null=True) + suggests = models.TextField(null=True) + size = models.IntegerField(null=False) + + class Meta: + unique_together = (("package", "version", "arch")) + + def __unicode__(self): + return self.package + +class PackageList(models.Model): + suite = models.ForeignKey(Suite) + package = models.ForeignKey(Package) + +class PackageContents(models.Model): + package = models.ForeignKey(Package) + filename = models.TextField(null=False) + +class Source(models.Model): + source = models.TextField(null=False) + srcversion = models.TextField(null=False) + binary = models.TextField(null=False) + architecture = models.TextField(null=False) + maintainer = models.TextField(null=False) + homepage = models.TextField(null=True) + short_description = models.TextField(null=True) + description = models.TextField(null=True) + vcs_browser = models.TextField(null=True) + vcs_repo = models.TextField(null=True) + build_depends = models.TextField(null=True) + directory = models.TextField(null=True) + copyright = models.TextField(null=True) + debchangelog = models.TextField(null=True) + changelog = models.TextField(null=True) + + class Meta: + unique_together = (("source", "srcversion")) + + def __unicode__(self): + return self.source + +class SourceList(models.Model): + suite = models.ForeignKey(Srcsuite) + source = models.ForeignKey(Source) + +# because file is a common identifier I use files as classname +class Files(models.Model): + files = models.TextField(null=False) + md5_hash = models.TextField(null=False) + size = models.IntegerField(null=False) + source = models.ForeignKey(Source) + + unitque_together = (("files", "md5_hash", "size"),) + diff --git a/pgweb/apt/sql/architecture.sql b/pgweb/apt/sql/architecture.sql new file mode 100644 index 0000000..397ba73 --- /dev/null +++ b/pgweb/apt/sql/architecture.sql @@ -0,0 +1 @@ +INSERT INTO apt_architecture VALUES ('all'); diff --git a/pgweb/apt/sql/package.sql b/pgweb/apt/sql/package.sql new file mode 100644 index 0000000..af9f8e4 --- /dev/null +++ b/pgweb/apt/sql/package.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS debversion; +ALTER TABLE apt_package ALTER COLUMN version type debversion; diff --git a/pgweb/apt/sql/packagecontents.sql b/pgweb/apt/sql/packagecontents.sql new file mode 100644 index 0000000..fbd9fd3 --- /dev/null +++ b/pgweb/apt/sql/packagecontents.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX apt_packagecontents_filename_gin ON apt_packagecontents USING gin (filename gin_trgm_ops); diff --git a/pgweb/apt/sql/pkg.sql b/pgweb/apt/sql/pkg.sql new file mode 100644 index 0000000..e7d20aa --- /dev/null +++ b/pgweb/apt/sql/pkg.sql @@ -0,0 +1,12 @@ +CREATE VIEW apt_pkg AS + SELECT s.release_id, + s.component_id, + s.architecture_id, + p.package, + p.version, + p.arch_id, + p.source, + p.srcversion + FROM apt_suite s + JOIN apt_packagelist pl ON s.id = pl.suite_id + JOIN apt_package p ON pl.package_id = p.id; diff --git a/pgweb/apt/sql/release.sql b/pgweb/apt/sql/release.sql new file mode 100644 index 0000000..6d49134 --- /dev/null +++ b/pgweb/apt/sql/release.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_release + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/source.sql b/pgweb/apt/sql/source.sql new file mode 100644 index 0000000..f6ab987 --- /dev/null +++ b/pgweb/apt/sql/source.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS debversion; +ALTER TABLE apt_source ALTER COLUMN srcversion TYPE debversion; diff --git a/pgweb/apt/sql/srcsuite.sql b/pgweb/apt/sql/srcsuite.sql new file mode 100644 index 0000000..f7a4627 --- /dev/null +++ b/pgweb/apt/sql/srcsuite.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_srcsuite + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/suite.sql b/pgweb/apt/sql/suite.sql new file mode 100644 index 0000000..bfcd3f6 --- /dev/null +++ b/pgweb/apt/sql/suite.sql @@ -0,0 +1,2 @@ +ALTER TABLE apt_suite + ALTER COLUMN active SET DEFAULT true; diff --git a/pgweb/apt/sql/vendor.sql b/pgweb/apt/sql/vendor.sql new file mode 100644 index 0000000..9e59bd9 --- /dev/null +++ b/pgweb/apt/sql/vendor.sql @@ -0,0 +1 @@ +INSERT INTO apt_vendor VALUES ('Debian'), ('Ubuntu'); diff --git a/pgweb/apt/utils.py b/pgweb/apt/utils.py new file mode 100644 index 0000000..73b17a9 --- /dev/null +++ b/pgweb/apt/utils.py @@ -0,0 +1,29 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch +# Christoph Berg + +from .models import Package, PackageList + +def build_related_list(rel_list, release): + result = [] + if rel_list: + for entry in rel_list.split(","): + entry = entry.strip() + l = [] + for subentry in entry.split("|"): + l.append(build_list_element(subentry, release)) + result.append(l) + return result + +def build_list_element(list_entry, release): + """Returns tuple (full string, package name, dependency package available in our archive)""" + list_entry = list_entry.strip() + s = list_entry.split(" ", 1) + pkg = s[0] + dep = '' + if len(s) > 1: + dep = s[1] + pkg_available = len(PackageList.objects.filter(\ + package__package=pkg, \ + suite__release=release)) + return (list_entry, pkg, dep, pkg_available) diff --git a/pgweb/apt/views.py b/pgweb/apt/views.py new file mode 100644 index 0000000..31cb7e5 --- /dev/null +++ b/pgweb/apt/views.py @@ -0,0 +1,373 @@ +# APT repository interface for the PostgreSQL website +# Authors: Adrian Vondendriesch +# Christoph Berg + +from pgweb.util.contexts import render_pgweb +from django.http import HttpResponse, Http404, HttpResponseRedirect +from django.db import connection +from .models import * +from .utils import * + +def apt(request): + return render_pgweb(request, 'apt', 'apt/apt.html', {}) + +def sources(request, release): + source_list_entries = SourceList.objects.filter(suite__release_id=release).order_by('source__source') + if not len(source_list_entries): + raise Http404 + + sources = [entry.source for entry in source_list_entries] + + return render_pgweb(request, 'apt', 'apt/sources.html', { + 'release': release, + 'sources': sources, + }) + +def source(request, package_name, release='sid-pgdg', component=None, filename=None): + + source_list_entries = SourceList.objects.filter(source__source__exact=package_name) + suites = [e.suite for e in source_list_entries] + + if release: + source_list_entries = source_list_entries.filter(suite__release_id__exact=release) + if component: + source_list_entries = source_list_entries.filter(suite__component_id__exact=component) + + sources = [s.source for s in source_list_entries] + + # check if at least one source package was found + if not len(sources): + raise Http404 + + source = sources[0] + current_component = source_list_entries[0].suite.component_id + current_release = source_list_entries[0].suite.release_id + + binaries = [binary.strip() for binary in source.binary.split(',')] + + if filename: + if filename == 'copyright': + title = 'Copyright file' + content = source.copyright + elif filename == 'debchangelog': + title = 'Debian changelog' + content = source.debchangelog + else: + title = 'Upstream changelog' + content = source.changelog + return render_pgweb(request, 'apt', 'apt/source_file.html', { + 'package': source, + 'filename': filename, + 'title': title, + 'content': content, + 'current_release': current_release, + 'current_component': current_component, + 'binaries': binaries, + }) + + source_list_entries = source_list_entries.filter(suite__release_id__exact=current_release) + components = [e.suite.component_id for e in source_list_entries] + + # create dependency list + build_dependencies = build_related_list(source.build_depends, current_release) + + return render_pgweb(request, 'apt', 'apt/source.html', { + 'package': source, + 'build_dependencies': build_dependencies, + 'distributions': suites, + 'current_release': current_release, + 'components': components, + 'current_component': current_component, + 'files': source.files_set.all(), + 'binaries': binaries + }) + +def binary_contents(request, package_name, release, component, arch): + + # fix contents for all packages + if arch == 'all': + arch = 'amd64' + + binary_list_entries = PackageList.objects.filter(package__package__exact=package_name, suite__release_id__exact=release, suite__component_id__exact=component, suite__architecture_id__exact=arch) + + if not len(binary_list_entries): + raise Http404 + + binary_list = binary_list_entries[0] + binary = binary_list.package + contents = binary.packagecontents_set.all().order_by("filename") + + return render_pgweb(request, 'apt', 'apt/binary_contents.html', { + 'package': binary, + 'current_release': release, + 'current_component': component, + 'contents': contents, + }) + +def binary(request, package_name, release='sid-pgdg', component='main'): + + binary_list_entries = PackageList.objects.filter(package__package__exact=package_name) + suites = [e.suite for e in binary_list_entries.distinct('suite__release')] + + if release: + binary_list_entries = binary_list_entries.filter(suite__release__exact=release) + components = [c.suite.component for c in binary_list_entries.distinct('suite__component')] + if component: + binary_list_entries = binary_list_entries.filter(suite__component_id__exact=component) + + # check if at least on source package was found + if not len(binary_list_entries): + raise Http404 + + binary = binary_list_entries[0].package + current_component = binary_list_entries[0].suite.component_id + current_release = binary_list_entries[0].suite.release_id + + dependencies = build_related_list(binary.depends, current_release) + recommendations = build_related_list(binary.recommends, current_release) + suggestions = build_related_list(binary.suggests, current_release) + + return render_pgweb(request, 'apt', 'apt/binary.html', { + 'package': binary, + 'dependencies': dependencies, + 'recommendations': recommendations, + 'distributions': suites, + 'current_release': current_release, + 'suggestions': suggestions, + 'components': components, + 'current_component': current_component, + 'files': None, + 'downloads': Package.objects.filter(package=package_name, version=binary.version) + }) + +def distributions(request): + releases = Release.objects.order_by("vendor", "relversion", "release") + + return render_pgweb(request, 'apt', 'apt/distributions.html', { + 'releases': releases, + }) + +def binaries(request, release, component_name=None, architecture_name=None): + + if not architecture_name: + architecture_name = 'amd64' + + suites = Suite.objects.filter(release=release, architecture=architecture_name) + components = [d.component for d in suites.distinct('component')] + + if not component_name: + component_name = 'main' + + suites = suites.filter(component_id=component_name) + + if not len(suites): + raise Http404 + + suite = suites[0] + + binary_lists = suite.packagelist_set.all().order_by('package__package') + binaries = [binary_list.package for binary_list in binary_lists] + + return render_pgweb(request, 'apt', 'apt/binaries.html', { + 'suite': suite, + 'components': components, + 'binaries': binaries + }) + +def search(request): + if 'package' in request.GET and request.GET['package']: + package = request.GET['package'] + + result_binaries = Package.objects.distinct('package').filter(package__icontains=package) + result_sources = Source.objects.distinct('source').filter(source__icontains=package) + + return render_pgweb(request, 'apt', 'apt/search.html', { + 'result': True, + 'package': package, + 'result_binaries': result_binaries, + 'result_sources': result_sources, + }) + + elif 'file' in request.GET and request.GET['file']: + filename = request.GET['file'] + + maxresults = 100 + result_filenames = PackageContents.objects.distinct('filename', 'package__package').filter(filename__contains=filename).order_by("filename")[:maxresults] + + return render_pgweb(request, 'apt', 'apt/search.html', { + 'filename': filename, + 'maxresults': maxresults, + 'result_filenames': result_filenames, + }) + + else: + return render_pgweb(request, 'apt', 'apt/search.html', { + }) + +def madison(request): + """Query interface compatible with rmadison(1)""" + + if 'package' in request.GET and request.GET['package']: + package = request.GET['package'] + + sql1 = """SELECT package, version, release_id, component_id, array_agg(architecture_id) AS architecture FROM ( + SELECT package, version, release_id, component_id, architecture_id + FROM apt_packagelist pl + JOIN apt_suite s ON (suite_id = s.id) + JOIN apt_package p ON (package_id = p.id) + UNION ALL SELECT source, srcversion, release_id, component_id, 'source' + FROM apt_sourcelist sl + JOIN apt_srcsuite ss ON (suite_id = ss.id) + JOIN apt_source s ON (source_id = s.id) + ORDER BY package, version, release_id, component_id, architecture_id + ) AS r WHERE """ + sql2 = "GROUP BY package, version, release_id, component_id" + + qual = [] + args = [package] + configdisplay = 'none' + + r = '' + if 'r' in request.GET: + r = 'checked' + qual.append('package ~ %s') + configdisplay = 'block' + else: + qual.append('package = %s') + + a = '' + if 'a' in request.GET and request.GET['a']: + a = request.GET['a'] + qual.append('architecture_id = %s') + args.append(a) + configdisplay = 'block' + + c = '' + if 'c' in request.GET and request.GET['c']: + c = request.GET['c'] + qual.append('component_id = %s') + args.append(c) + configdisplay = 'block' + + s = '' # distributions are called suites in dak + if 's' in request.GET and request.GET['s']: + s = request.GET['s'] + qual.append('release_id = %s') + args.append(s) + configdisplay = 'block' + + cursor = connection.cursor() + cursor.execute(sql1 + " AND ".join(qual) + sql2, args) + return_set = cursor.fetchall() + + plen = 1 + vlen = 1 + rlen = 1 + for (pkg, version, release, component, architecture) in return_set: + if component != 'main': + release += '/' + component + if len(pkg) > plen: + plen = len(pkg) + if len(version) > vlen: + vlen = len(version) + if len(release) > rlen: + rlen = len(release) + fstr = " %%-%ds | %%-%ds | %%-%ds | %%s\n" % (plen, vlen, rlen) + + content = '' + for (pkg, version, release, component, architecture) in return_set: + if component != 'main': + release += '/' + component + content += fstr % (pkg, version, release, ", ".join(architecture)) + + if 'text' in request.GET: + return HttpResponse(content, content_type='text/plain') + + return render_pgweb(request, 'apt', 'apt/madison.html', { + 'result': True, + 'package': package, + 'r': r, + # no need to pass checkbox 'text' value here + 'a': a, + 'c': c, + 's': s, + 'configdisplay': configdisplay, + 'content': content, + }) + + else: + return render_pgweb(request, 'apt', 'apt/madison.html', { + 'configdisplay': 'none', + }) + +def qa(request, query=None): + result = None + + if True or query == None: # TODO: fix the SQL queries + description = "APT Repository Quality Assurance" + sql = None + + elif query == "binary_missing_on_architecture": + description = "Binary package versions that exist only on one architecture" + sql = """select * from apt_pkg a where not exists + (select * from apt_pkg b where a.package = b.package and a.version = b.version and a.architecture_id <> b.architecture_id) + order by release_id, package, version""" + + elif query == "wrong_tag_in_version": + description = "Package versions with wrong pgdg tag" + sql = """select p.* from apt_pkg p join apt_release r on (p.release_id = r.release) + where version !~ case + when release = 'sid-pgdg' then 'pgdg' + when release = 'lenny-pgdg' then 'pg(dg|apt)50' + when release = 'etch-pgdg' then 'pg(dg|apt)40' + when vendor_id = 'Debian' then 'pgdg'||relversion||0 + when release = 'precise-pgdg' then 'pgdg12.4' + when release = 'lucid-pgdg' then 'pgdg10.4' + when vendor_id = 'Ubuntu' then 'pgdg'||relversion + end + and r.active + order by release_id, package, version, architecture_id""" + + elif query == "binary_without_source": + description = "Binary packages without source" + sql = """select * from apt_pkg p where not exists + (select * from apt_source s where (p.source, p.srcversion) = (s.source, s.srcversion)) + order by release_id, package, version, architecture_id""" + + elif query == "binary_in_sid_only": + description = "Binary packages in sid-pgdg missing in other suites" + sql = """SELECT DISTINCT sidpkg.package, sidpkg.version, sidpkg.source, sidpkg.srcversion, + rel.release AS missing_release FROM apt_pkg sidpkg + JOIN apt_release rel ON (true) + LEFT JOIN apt_pkg p ON (rel.release = p.release_id and sidpkg.package = p.package) + WHERE sidpkg.release_id = 'sid-pgdg' AND rel.release <> 'sid-pgdg' AND rel.active + AND p.package IS NULL + ORDER BY sidpkg.package, missing_release""" + + elif query == "binary_with_old_version": + description = "Binary packages with versions different from sid-pgdg" + sql = """SELECT sidpkg.release_id, sidpkg.component_id, sidpkg.package, sidpkg.version, + sidpkg.architecture_id, p.release_id, p.version FROM apt_pkg sidpkg + JOIN apt_pkg p ON (sidpkg.package = p.package AND sidpkg.component_id = p.component_id AND sidpkg.architecture_id = p.architecture_id) + WHERE sidpkg.release_id = 'sid-pgdg' + AND regexp_replace(sidpkg.version, '.pgdg.*', '') <> regexp_replace(p.version, '.pgdg.*', '') + ORDER BY sidpkg.component_id, sidpkg.package, sidpkg.architecture_id, p.release_id""" + + elif query == "source_without_binary": + description = "Source packages without binaries" + sql = """SELECT source, srcversion FROM apt_source s WHERE NOT EXISTS + (SELECT * FROM apt_pkg p WHERE (s.source, s.srcversion) = (p.source, p.srcversion))""" + + else: + raise Http404 + + if sql: + cursor = connection.cursor() + cursor.execute(sql) + result = cursor.fetchall() + + return render_pgweb(request, 'apt', 'apt/qa.html', { + 'query': query, + 'description': description, + 'result': result, + }) diff --git a/pgweb/settings.py b/pgweb/settings.py index 540a099..15e72d9 100644 --- a/pgweb/settings.py +++ b/pgweb/settings.py @@ -115,6 +115,7 @@ INSTALLED_APPS = [ 'pgweb.featurematrix', 'pgweb.search', 'pgweb.pugs', + 'pgweb.apt', ] # Default format for date/time (as it changes between machines) @@ -161,5 +162,7 @@ FRONTEND_SMTP_RELAY = "magus.postgresql.org" # Where to relay use OAUTH = {} # OAuth providers and keys PGDG_ORG_ID = -1 # id of the PGDG organisation entry +APT_DIR = "/srv/apt/pub/repos/apt" # Directory containing the dists/ dir from apt.postgresql.org + # Load local settings overrides from .settings_local import * diff --git a/pgweb/urls.py b/pgweb/urls.py index d40673f..55ddcbe 100644 --- a/pgweb/urls.py +++ b/pgweb/urls.py @@ -3,6 +3,7 @@ from django.views.generic import RedirectView from pgweb.util.signals import register_basic_signal_handlers +import pgweb.apt.views import pgweb.contributors.views import pgweb.core.views import pgweb.docs.views @@ -110,6 +111,37 @@ urlpatterns = [ url(r'^sitemap.xml', pgweb.core.views.sitemap), url(r'^sitemap_internal.xml', pgweb.core.views.sitemap_internal), + ### + # Packages + ### + url(r'^repos/apt/$', pgweb.apt.views.apt), + url(r'^repos/apt/dists/$', pgweb.apt.views.distributions), + + url(r'^repos/apt/sources/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/sources/(?P[\w.+-]+)/$', pgweb.apt.views.sources), + + url(r'^repos/apt/source/$', RedirectView.as_view(url='/repos/apt/sources', permanent=True)), + url(r'^repos/apt/source/(?P[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P[\w.+-]+)/(?P[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P[\w.+-]+)/(?P[\w.+-]+)/(?Pcopyright|debchangelog|changelog)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P[\w.+-]+)/(?P[\w.+-]+)/(?P[\w.+-]+)/$', pgweb.apt.views.source), + url(r'^repos/apt/source/(?P[\w.+-]+)/(?P[\w.+-]+)/(?P[\w.+-]+)/(?Pcopyright|debchangelog|changelog)/$', pgweb.apt.views.source), + + url(r'^repos/apt/binaries/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/binaries/(?P[\w.+-]+)/$', pgweb.apt.views.binaries), + url(r'^repos/apt/binaries/(?P[\w.+-]+)/(?P[\w\-\.]+)/$', pgweb.apt.views.binaries), + + url(r'^repos/apt/binary/$', RedirectView.as_view(url='/repos/apt/dists/', permanent=True)), + url(r'^repos/apt/binary/(?P[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P[\w.+-]+)/(?P[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P[\w.+-]+)/(?P[\w.+-]+)/(?P[\w.+-]+)/$', pgweb.apt.views.binary), + url(r'^repos/apt/binary/(?P[\w.+-]+)/(?P[\w.+-]+)/(?P[\w.+-]+)/(?P[\w.+-]+)/contents/$', pgweb.apt.views.binary_contents), + + url(r'^repos/apt/search/$', pgweb.apt.views.search), + url(r'^repos/apt/madison/$', pgweb.apt.views.madison), # machine-readable search interface + url(r'^repos/apt/qa/$', pgweb.apt.views.qa), + url(r'^repos/apt/qa/(?P[\w.+-]+)/$', pgweb.apt.views.qa), + ### # Workaround for broken links pushed in press release ### diff --git a/pgweb/util/contexts.py b/pgweb/util/contexts.py index 43f6c3e..b93cacd 100644 --- a/pgweb/util/contexts.py +++ b/pgweb/util/contexts.py @@ -83,6 +83,15 @@ sitenav = { {'title': 'Change password', 'link': '/account/changepwd/'}, {'title': 'Logout', 'link': '/account/logout'}, ], + 'apt': [ + {'title': 'APT Repository', 'link': '/repos/apt/'}, + {'title': 'Distributions', 'link': '/repos/apt/dists/'}, + {'title': 'Search', 'link': '/repos/apt/search/', 'submenu': [ + {'title': 'Query API', 'link': '/repos/apt/madison/'}, + {'title': 'Repository QA', 'link': '/repos/apt/qa/'}, + ]}, + {'title': 'Wiki', 'link': 'http://wiki.postgresql.org/wiki/Apt'}, + ], } diff --git a/templates/apt/apt.html b/templates/apt/apt.html new file mode 100644 index 0000000..7cfc55a --- /dev/null +++ b/templates/apt/apt.html @@ -0,0 +1,80 @@ +{%extends "base/page.html"%} +{%block title%}Apt Repository{%endblock%} +{%block contents%} +

Apt Repository with PostgreSQL Packages for Debian and Ubuntu

+ +

+The PostgreSQL Global Development Group (PGDG) maintains an APT repository of +PostgreSQL packages for Debian and Ubuntu located at +http://apt.postgresql.org/pub/repos/apt/. We aim at building PostgreSQL server +packages as well as extensions and modules packages on several Debian/Ubuntu +releases for all PostgreSQL versions supported. +

+ +

+Currently, we support + +

    +
  • Debian 6.0 (squeeze), 7.0 (wheezy), and unstable (sid) 64/32 bit (amd64/i386)
    • +
  • Ubuntu 10.04 (lucid) and 12.04 (precise) 64/32 bit (amd64/i386)
    • +
  • PostgreSQL 8.3, 8.4, 9.0, 9.1, 9.2
    • +
  • Several server extensions such as Slony-I, various PL languages, and datatypes
    • +
  • Applications like pgadmin3, pgbouncer, and pgpool-II
    • +
+

+ +

+Packages for older PostgreSQL versions and older Debian distributions will +continue to stay in the repository; updates for those will be provided on an +ad-hoc basis. +

+ + + +

Using the Apt Repository

+

+If the version included in your version of Debian/Ubuntu is not the one you want, +you can use the PostgreSQL Apt Repository. This repository will integrate +with your normal systems and patch management, and provide automatic +updates for all supported versions of PostgreSQL throughout the support +lifetime of PostgreSQL. +

+

+To use the apt repository, follow these steps: +

+
    +
  • + +
    • +
  • Create the file /etc/apt/sources.list.d/pgdg.list, and add a line + for the repository + deb http://apt.postgresql.org/pub/repos/apt/ OS_VERSION-pgdg main +
    • +
  • Import the repository signing key, and update the package lists + +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \
    +  sudo apt-key add -
    +sudo apt-get update +     +
    • +
+

+For more information about the apt repository, including answers to frequent +questions, please see the apt page on +the wiki. +

+ +{%endblock%} diff --git a/templates/apt/binaries.html b/templates/apt/binaries.html new file mode 100644 index 0000000..e6536c2 --- /dev/null +++ b/templates/apt/binaries.html @@ -0,0 +1,46 @@ +{%extends "base/page.html"%} +{%block title%}Binary Packages in {{suite.release_id}}/{{suite.component_id}}{%endblock%} +{%block contents%} +
+
+

Binary Packages in {{suite.release_id}}/{{suite.component_id}} ({{binaries|length}})

+ +

+

    +{% for binary in binaries %} +
  • + {{binary.package}} + {{binary.version}}
    + {{binary.short_description}} +
    • +{% endfor %} +
+

+
+
+ +{% if components|length > 1 %} +
+
+
+
Other Components
+
+
    + {% for component in components %} +
  • + {% if component.component != suite.component_id %} + {{component}} + {% else %} + {{component}} + {% endif %} +
    • + {% endfor %} +
+
+
+
+
+{% endif %} + +{%endblock%} diff --git a/templates/apt/binary.html b/templates/apt/binary.html new file mode 100644 index 0000000..32f0b2e --- /dev/null +++ b/templates/apt/binary.html @@ -0,0 +1,167 @@ +{%extends "base/page.html"%} +{%block title%}Package {{package.package}} ({{package.version}}) in {{current_release}}{% if current_component != "main" %}/{{current_component}}{% endif %}{%endblock%} +{%block contents%} +
+
+

{{package.package}} ({{package.version}}) in {{current_release}}{% if current_component != "main" %}/{{current_component}}{% endif %}

+ +

+{{package.short_description}} +

+ +

+{% autoescape off %} +{{package.description}} +{% endautoescape %} +

+ +{% if package.homepage %} +

Homepage: + {{package.homepage}} +{% endif %} + +

+Maintainer: {{ package.maintainer }} +

+ +{% if package.vcs_browser or package.vcs_repo%} +

VCS

+ + {% if package.vcs_browser %} +

Browser

+

+ {{package.vcs_browser}} +

+ {% endif %} + + {% if package.vcs_repo %} +

Repository

+

+ {{package.vcs_repo}} +

+ {% endif %} +{% endif %} + +{% if dependencies or recommendations or suggestions %} +

Related Packages

+{% if dependencies %} +

Dependencies {% if package.arch_id != "all" %} (on {{package.arch_id}}) {% endif %}

+
    + {% for dep in dependencies %} + {% include "apt/dependency.html" %} + {% endfor %} +
+{% endif %} + +{% if recommendations %} +

Recommendations

+
    + {% for dep in recommendations %} + {% include "apt/dependency.html" %} + {% endfor %} +
+{% endif %} + +{% if suggestions %} +

Suggestions

+
    + {% for dep in suggestions %} + {% include "apt/dependency.html" %} + {% endfor %} +
+{% endif %} +{% endif %} + +

Download {{package.package}} for {{current_release}}

+
+ + + + + + + + {% for download in downloads %} + + + + + + + {% endfor %} +
ArchitecturePackage SizeInstalled SizeFile List
+ {{download.arch_id}} + {{download.size|filesizeformat}}{{download.installed_size|filesizeformat}}File List
+
+
+
+ + + + +{% if components|length > 1 %} +
+
+
+
This Package in other Components
+
+
    +{% for comp in components %} + {% if comp.component == current_component %} +
  • {{comp}}
    • + {% else %} +
  • + {{comp}} +
    • + {% endif %} +{% endfor %} +
+
+
+
+
+{% endif %} + +
+
+
+
Distributions
+
+
    + {% for dist in distributions %} +
  • + {% if current_release and current_release == dist.release_id %} + {{dist.release_id}} + {% else %} + {{dist.release_id}} + {% endif %} +
    • + {% endfor %} +
+
+
+
+
+ +{%endblock%} diff --git a/templates/apt/binary_contents.html b/templates/apt/binary_contents.html new file mode 100644 index 0000000..c42923b --- /dev/null +++ b/templates/apt/binary_contents.html @@ -0,0 +1,14 @@ +{%extends "base/page.html"%} +{%block title%}File List of Package {{package.package}} ({{package.version}} {{package.arch.architecture}}){%endblock%} +{%block contents%} +
+
+

{{package.package}} ({{package.version}} {{package.arch.architecture}}) in {{current_release}} File List

+

+ +{% for file in contents %}{{file.filename}}
+{% endfor %} + +

+ +{%endblock%} diff --git a/templates/apt/dependency.html b/templates/apt/dependency.html new file mode 100644 index 0000000..c7f0709 --- /dev/null +++ b/templates/apt/dependency.html @@ -0,0 +1,12 @@ +
  • +{% for alternative in dep %} + {% if alternative.3 %} + {{alternative.1}} + {{alternative.2}} + {% else %} + {{alternative.0}} + {% endif %} + {% if alternative != dep|last %} | {% endif %} +{% endfor %} +
    • diff --git a/templates/apt/distributions.html b/templates/apt/distributions.html new file mode 100644 index 0000000..82ff46f --- /dev/null +++ b/templates/apt/distributions.html @@ -0,0 +1,20 @@ +{%extends "base/page.html"%} +{%block title%}Distributions on apt.postgresql.org{%endblock%} +{%block contents%} +

    Distributions

    +

    +

    + + {% for release in releases %} + + + + + + + {% endfor %} +
    {{release.vendor_id}} {{release.relversion}}{{release.release}}BinariesSources
    +
    +

    + +{%endblock%} diff --git a/templates/apt/madison.html b/templates/apt/madison.html new file mode 100644 index 0000000..3709a14 --- /dev/null +++ b/templates/apt/madison.html @@ -0,0 +1,50 @@ +{%extends "base/page.html"%} +{%block title%}Query apt.postgresql.org{%endblock%} +{%block contents%} + + + +

    Query apt.postgresql.org

    + +
    + +
    + Enter a package name + (Click for more options)
    +
    + + + + + + +
    Regexp search
    Text output
    Architecture
    Component
    Distribution
    +
    +
    + + +{% if package %} +

    Results for "{{package}}"

    + +
    {{content}}
    + +{% else %} + +

    +A command line client for this page is +rmadison(1) +contained in the +devscripts package. +

    +{% endif %} + +{%endblock%} diff --git a/templates/apt/qa.html b/templates/apt/qa.html new file mode 100644 index 0000000..fcf94b2 --- /dev/null +++ b/templates/apt/qa.html @@ -0,0 +1,33 @@ +{%extends "base/page.html"%} +{%block title%}{{description}}{%endblock%} +{%block contents%} +
    +
    +

    {{description}}

    + +{% if query %} + + +{% for row in result %} + + {% for column in row %} + + {% endfor %} + +{% endfor %} +
    {{column}}
    + +{% else %} + + + +{% endif %} + +{%endblock%} diff --git a/templates/apt/search.html b/templates/apt/search.html new file mode 100644 index 0000000..fbeb67b --- /dev/null +++ b/templates/apt/search.html @@ -0,0 +1,87 @@ +{%extends "base/page.html"%} +{%block title%}Search apt.postgresql.org{%endblock%} +{%block contents%} +

    Search on apt.postgresql.org

    + +

    Search for Packages

    + +
    + + +
    + +Enter a package name or substring + +{% if package %} +

    Results for "{{package}}"

    + +
    +

    Binary Packages: {{result_binaries|length}}

    + {% if result_binaries %} +
    + + {% for binary in result_binaries %} +
    {{binary.package}}
    +
    {{binary.short_description}}
    + {% endfor %} + +
    + {% else %} +

    + No packages found. +

    +
    + {% endif %} + +
    +

    Source Packages: {{result_sources|length}}

    + {% if result_sources %} +
    + + {% for source in result_sources %} +
    {{source.source}}
    +
    {{source.short_description}}
    + {% endfor %} + +
    + {% else %} +

    + No packages found. +

    +
    + {% endif %} +{% endif %} + +

    Search for Files in Packages

    + +
    + + +
    + +Enter a filename or substring + +{% if filename %} +

    Results for "{{filename}}"

    + +
    + {% if result_filenames %} + {% if result_filenames|length >= maxresults %} +

    Files: Too many results, only {{maxresults}} will be shown

    + {% else %} +

    Files: {{result_filenames|length}}

    + {% endif %} + + {% for file in result_filenames %} + + {% endfor %} +
    {{file.filename}}{{file.package}}
    + {% else %} +

    + No packages found. +

    + {% endif %} +
    +{% endif %} + +{%endblock%} diff --git a/templates/apt/source.html b/templates/apt/source.html new file mode 100644 index 0000000..cf9768f --- /dev/null +++ b/templates/apt/source.html @@ -0,0 +1,136 @@ +{%extends "base/page.html"%} +{%block title%}Source Package {{package.source}} ({{package.srcversion}}) in {{current_release}}{%endblock%} +{%block contents%} +
    +
    +

    {{package.source}} ({{package.srcversion}}) in {{current_release}}

    + +

    +{{package.short_description}} +

    + +

    +{% autoescape off %} +{{package.description}} +{% endautoescape %} +

    + +

    +Maintainer: {{ package.maintainer }} +

    + +{% if package.homepage %} +

    Homepage: + {{package.homepage}} +{% endif %} + +

    Files

    +
      + {% for f in files %} +
    • + {{f.files}}
      + {{f.size|filesizeformat}}, {{f.md5_hash}} +
      • + {% endfor %} +
    + +{% if build_dependencies %} +

    Build Dependencies

    +
      + {% for dep in build_dependencies %} + {% include "apt/dependency.html" %} + {% endfor %} +
    +{% endif %} + +
    +
    + +
    +
    +
    +
    Binaries
    +
    +
      + {% for binary in binaries %} +
    • + {{binary}}
      • + {% endfor %} +
    +
    +
    +
    +
    + +
    +
    +
    +
    Resources
    +
    + +
    +
    +
    +
    + +
    +
    +
    +
    Distributions
    +
    +
      + {% for dist in distributions %} +
    • + {% if current_release and current_release == dist.release_id %} + {{dist.release_id}} + {% else %} + {{dist.release_id}} + {% endif %} +
      • + {% endfor %} +
    +
    +
    +
    +
    + +{% if components|length > 1 %} +
    +
    +
    +
    Components
    +
    +
      +{% for comp in components %} + {% if comp == current_component %} +
    • {{comp}}
      • + {% else %} +
    • + {{comp}} +
      • + {% endif %} +{% endfor %} +
    +
    +
    +
    +
    +{% endif %} + +{%endblock%} diff --git a/templates/apt/source_file.html b/templates/apt/source_file.html new file mode 100644 index 0000000..617b03c --- /dev/null +++ b/templates/apt/source_file.html @@ -0,0 +1,66 @@ +{%extends "base/page.html"%} +{%block title%}{{title}} for {{package.source}} ({{package.srcversion}}) in {{current_release}}{%endblock%} +{%block contents%} +
    +
    +

    {{title}} for {{package.source}} ({{package.srcversion}}) in {{current_release}}

    + +

    +{{package.short_description}} +

    + +
    +{{content}}
+
    + +
    +
    + +
    +
    +
    +
    Source
    +
    +
      +
    • {{package.source}}
      • + + {% if filename == "copyright" %} +
    • Copyright file
      • + {% else %} +
    • Copyright file
      • + {% endif %} + + {% if filename == "debchangelog" %} +
    • Package changelog
      • + {% else %} +
    • Package changelog
      • + {% endif %} + + {% if filename == "changelog" %} +
    • Upstream changelog
      • + {% else %} +
    • Upstream changelog
      • + {% endif %} +
    +
    +
    +
    +
    + +
    +
    +
    +
    Binaries
    +
    +
      + {% for binary in binaries %} +
    • + {{binary}}
      • + {% endfor %} +
    +
    +
    +
    +
    + +{%endblock%} diff --git a/templates/apt/sources.html b/templates/apt/sources.html new file mode 100644 index 0000000..11c83f3 --- /dev/null +++ b/templates/apt/sources.html @@ -0,0 +1,18 @@ +{%extends "base/page.html"%} +{%block title%}Source Packages in {{release}}{%endblock%} +{%block contents%} +

    Source Packages in {{release}} ({{sources|length}})

    + +

    +

      + {% for source in sources %} +
    • + {{source.source}} + {{source.srcversion}}
      + {{source.short_description}} +
      • + {% endfor %} +
    +

    + +{%endblock%} diff --git a/tools/apt/import-packagelist.py b/tools/apt/import-packagelist.py new file mode 100755 index 0000000..4b1973e --- /dev/null +++ b/tools/apt/import-packagelist.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python + +# debmirror -v -h apt.postgresql.org -r pub/repos/apt --method=http -d sid-pgdg,buster-pgdg --omit-suite-symlinks -s main,9.3,9.4,9.5,9.6,10,11,12 -a source,amd64,i386,ppc64el --exclude='\.deb$' --getcontents --no-check-gpg /srv/repo + +import apt_pkg, markdown, os, re, sys, time, gzip, subprocess + +# Set up for accessing django +import django +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local") +import sys +sys.path.append('../../pgweb') +from django.conf import settings +#from apt import models as c + +#from core.models import ImportedRSSFeed, ImportedRSSItem +from django.db import transaction, connection + +#transaction.enter_transaction_management() +#transaction.managed() + +config = { + 'FILTER': [ + 'testing', + 'deprecated' + ] + } + +running_config = { + 'VERBOSE': 0, + 'FORCE': False, + 'DISTONLY': False, + } + +usage = """ +%s [options] + -h show this help + -v verbose + -f force (deletes every package before insert) + -d ignore errors from dscextract (useful for running on dists/ only) +""" % sys.argv[0] + +def dir_filter(string): + for f in config['FILTER']: + if f in string: + return 0 + return 1 + +def getArchitectureNames(directories): + binary_pattern = 'binary-' + binaries = [] + for directory in directories: + if re.match(binary_pattern, directory): + binaries.append(directory.split('-')[1]) + + return binaries + +def buildFileDict(flist): + fdict = {} + for line in flist: + columns = line.split('\t') + if len(columns) > 1 and columns[0] != 'FILE': + filename = '/' + columns[0] + packagename = columns[1].split('/')[-1].strip() + try: + fdict[packagename] + except: + fdict[packagename] = [] + fdict[packagename].append(filename) + #print "P-Name:", packagename + return fdict + +def main(*args, **kwargs): + running_config['VERBOSE'] = 0 + + # args + if len(args) > 0: + for arg in args[0]: + if arg == '-v': + running_config['VERBOSE'] += 1 + if arg == '-f': + running_config['FORCE'] = True + if arg == '-d': + running_config['DISTONLY'] = True + if arg == '-h': + print usage + exit(0) + + cursor = connection.cursor() + + # releases + base_dir = getattr(settings, 'APT_DIR') + "/dists/" + releases = [directory for directory in os.listdir(base_dir) \ + if os.path.isdir(os.path.join(base_dir, directory))] + releases = filter(dir_filter, releases) + if running_config['VERBOSE'] > 0: + print "Releases to scan (%s): %s" % \ + (len(releases), ', '.join(releases)) + + for release in releases: + print "%s:" % (release) + cur_dir = base_dir + release + components = [component for component in os.listdir(cur_dir) \ + if os.path.isdir(os.path.join(cur_dir, component))] + components = filter(dir_filter, components) + if running_config['VERBOSE'] > 0: + print "\tComponents in %s to scan (%s): %s" % \ + (release, len(components), ', '.join(components)) + + for component in components: + if running_config['VERBOSE'] > 0: + print "\t%s:" % (component) + + # insert components, that aren't not yet registered + sql = """ + INSERT INTO apt_component + SELECT * FROM (VALUES (%s)) comp(component) + WHERE NOT EXISTS ( + SELECT * FROM apt_component + WHERE component = comp.component + ) + """ + cursor.execute(sql, (component,)) + + srcsuite_id, last_update = insertSuite(release, component) + parseFile(last_update, srcsuite_id, release, component) + + # read binary architectures + architectures = getArchitectureNames(os.listdir(os.path.join(cur_dir, component))) + + for architecture in architectures: + # insert architectures, that aren't not yet registered + sql = """ + INSERT INTO apt_architecture + SELECT * FROM (VALUES (%s)) arch(architecture) + WHERE NOT EXISTS ( + SELECT * FROM apt_architecture + WHERE architecture = arch.architecture + ) + """ + cursor.execute(sql, (architecture,)) + + suite_id, last_update = insertSuite(release, component, architecture) + parseFile(last_update, suite_id, release, component, architecture) + + # update source package descriptions + sql = """UPDATE apt_source s SET + short_description = (SELECT short_description FROM apt_package p WHERE (p.source, p.srcversion) = (s.source, s.srcversion) ORDER BY source = p.package DESC, p.package LIMIT 1), + description = (SELECT description FROM apt_package p WHERE (p.source, p.srcversion) = (s.source, s.srcversion) ORDER BY source = p.package DESC, p.package LIMIT 1) + WHERE s.short_description IS NULL""" + cursor.execute(sql) + transaction.commit() + + connection.close() + return + + +def insertSuite(release, component, architecture=None): + cursor = connection.cursor() + + sql = "SELECT release FROM apt_release WHERE release = %s" + cursor.execute(sql, (release,)) + if not cursor.fetchone(): + sql = "INSERT INTO apt_release (vendor_id, release, relversion, active) VALUES ('Debian', %s, '', true)" + cursor.execute(sql, (release,)) + + if architecture is None: + tbl = "srcsuite" + src = True + else: + tbl = "suite" + src = False + + # check if srcsuite is already in db + sql = "SELECT id, extract('epoch' from last_update) FROM apt_%s WHERE " % (tbl) + if src: + sql = sql + "(release_id, component_id) = (%s, %s) " + args_tpl = (release, component) + else: + sql = sql + "(release_id, component_id, architecture_id) = (%s, %s, %s)" + args_tpl = (release, component, architecture) + cursor.execute(sql, args_tpl) + + return_set = cursor.fetchone() + + if return_set: + suite_id = return_set[0] + last_update = return_set[1] + + else: + last_update = None + sql = "INSERT INTO apt_%s " % (tbl) + if src: + sql = sql + "(release_id, component_id) " + sql = sql + "VALUES (%s, %s) " + args_tpl = (release, component) + else: + sql = sql + "(release_id, component_id, architecture_id) " + sql = sql + "VALUES (%s, %s, %s) " + args_tpl = (release, component, architecture) + sql = sql + "RETURNING id" + cursor.execute(sql, args_tpl) + + suite_id = cursor.fetchone() + + if running_config['VERBOSE'] and not last_update: + if src: + print "\tinserted new source suite (%s, %s)" % (release, component) + else: + print "\tinserted new suite (%s, %s, %s)" % (release, component, architecture) + + return suite_id, last_update; + +def parseFile(last_update, suite_id, release, component, architecture=None): + verbose = running_config['VERBOSE'] + cursor = connection.cursor() + packagesfile = "" + source_re = re.compile('(.*) \((.*)\)') + #binnmu_re = re.compile('\+b\d+$') + dscname_re = re.compile(' ([^ ]*\.dsc)') + + if architecture is None: + src = True + tbl = 'source' + + packagesfile = settings.APT_DIR + '/dists/%s/%s/source/Sources.gz' \ + % (release, component) + else: + src = False + tbl = 'package' + + packagesfile = settings.APT_DIR + '/dists/%s/%s/binary-%s/Packages.gz' \ + % (release, component, architecture) + + if not os.path.isfile(packagesfile): + raise Exception("%s not found" % (packagesfile,)) + + #check if an update is necessary + mtime = os.path.getmtime(packagesfile) + if last_update and mtime <= last_update + 1 and not running_config['FORCE']: # allow 1s offset for microsecond timestamps + #if verbose: + # print "\t\t%s is uptodate, skipping" % (packagesfile,) + return + + if verbose: + print "\t\tRead file %s" % (packagesfile,) + + if not src: + contents_plain = gzip.open(settings.APT_DIR + '/dists/%s/%s/Contents-%s.gz' \ + % (release, component, architecture)) + contents = buildFileDict(contents_plain) + + sql = "DELETE FROM apt_%slist " % (tbl) + sql = sql + "WHERE suite_id = %s" + cursor.execute(sql, (suite_id,)) + + content = os.popen("zcat '%s'" % (packagesfile)) + package = apt_pkg.TagFile(content) + + # for each package + while package.step(): + insert = True + package_content = {} + package_content['Package'] = package.section.get('Package') + package_content['Version'] = package.section.get('Version') + package_content['Binary'] = package.section.get('Binary') + package_content['Maintainer'] = package.section.get('Maintainer') + if package.section.get('Uploaders'): + package_content['Maintainer'] += ', ' + package.section.get('Uploaders') + package_content['Architecture'] = package.section.get('Architecture') + if package.section.get('Description'): + package_content['Short-Description'] = package.section.get('Description').split('\n')[0] + description = '' + list_needs_newline = False + list_started = False + for line in package.section.get('Description').split('\n ')[1:]: + if line == '.': + line = '' + list_needs_newline = False + elif line[0:3] == ' * ' or line[0:3] == ' - ': + # on seeing a new bullet list, make sure there is a newline before it + if list_needs_newline: + description += '\n' + line = '*' + line[2:] + list_needs_newline = False + list_started = True + elif not list_started: + list_needs_newline = True + description += line + '\n' + package_content['Description'] = markdown.markdown(description.decode("utf-8"), tab_length=1) + package_content['Homepage'] = package.section.get('Homepage') + package_content['VCS-Browser'] = package.section.get('VCS-Browser') + package_content['VCS-Repo'] = package.section.get('VCS-Svn') + if not package_content['VCS-Repo']: + package_content['VCS-Repo'] = package.section.get('VCS-Git') + if not package_content['VCS-Repo']: + package_content['VCS-Repo'] = package.section.get('VCS-Bzr') + package_content['Build-Depends'] = package.section.get('Build-Depends') + package_content['Build-Depends-Indep'] = package.section.get('Build-Depends-Indep') + if package_content['Build-Depends-Indep']: + package_content['Build-Depends'] += ", %s" % package_content['Build-Depends-Indep'] + package_content['Depends'] = package.section.get('Depends') + package_content['Recommends'] = package.section.get('Recommends') + package_content['Suggests'] = package.section.get('Suggests') + package_content['Directory'] = package.section.get('Directory') + package_content['Filename'] = package.section.get('Filename') + package_content['Size'] = package.section.get('Size') + if package.section.has_key('Installed-Size'): + package_content['Installed-Size'] = int(package.section.get('Installed-Size')) * 1024 + package_content['Files'] = package.section.get('Files') + + sourcefield = package.section.get('Source') + if sourcefield: + match = source_re.match(sourcefield) + if match: + package_content['Source'], package_content['Source-Version'] = \ + match.group(1), match.group(2) + else: + package_content['Source'], package_content['Source-Version'] = \ + sourcefield, package_content['Version'] + else: + package_content['Source'], package_content['Source-Version'] = \ + package_content['Package'], package_content['Version'] + + if src: + match = dscname_re.search(package_content['Files']) + if not match: + raise Exception('Files section without .dsc file') + dscfile = settings.APT_DIR + '/' + package_content['Directory'] + '/' + match.group(1) + + package_content['Copyright'] = None + try: + package_content['Copyright'] = subprocess.check_output( + ['dscextract', '-f', dscfile, 'debian/copyright']) + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + package_content['Debchangelog'] = None + try: + package_content['Debchangelog'] = subprocess.check_output( + ['dscextract', '-f', dscfile, 'debian/changelog']) + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + package_content['Changelog'] = None + try: + package_content['Changelog'] = None # TODO + except subprocess.CalledProcessError as e: + if e.returncode != 1 and not running_config['DISTONLY']: + raise + + # check if the package already exists + sql = "SELECT id FROM apt_%s WHERE " % (tbl) + if src: + sql = sql + "(source, srcversion) = (%(Package)s, %(Version)s)" + else: + sql = sql + "(package, version, arch_id) = (%(Package)s, %(Version)s, %(Architecture)s)" + cursor.execute(sql, package_content) + found = cursor.fetchone() + + if found: + package_id = found[0] + if running_config['FORCE']: + if verbose: + print "\t\tPackage %(Package)s already exists, force delete" % package_content + + # delete list entries + if src: + sql = "DELETE FROM apt_sourcelist WHERE source_id = %s" + else: + sql = "DELETE FROM apt_packagelist WHERE package_id = %s" + cursor.execute(sql, (package_id,)) + + # delete files entries + if src: + sql = "DELETE FROM apt_files WHERE source_id = %s" + cursor.execute(sql, (package_id,)) + + sql = "DELETE FROM apt_%s " % (tbl) + "WHERE id = %s" + cursor.execute(sql, (package_id,)) + insert = True + else: + #if verbose: + # print "Package %(Package)s already exists and up to date, continue" % package_content + insert = False + + if insert: + if verbose: + print "\t\tFound new Package %(Package)s (%(Version)s %(Architecture)s), inserting" % package_content + + # isert new package + sql = "INSERT INTO apt_%s " % (tbl) + if src: + sql = sql + "(source, srcversion, \"binary\", architecture, maintainer, homepage, vcs_browser, vcs_repo, build_depends, directory, copyright, debchangelog, changelog) " + sql = sql + "VALUES (%(Package)s, %(Version)s, %(Binary)s, %(Architecture)s, %(Maintainer)s, %(Homepage)s, %(VCS-Browser)s, %(VCS-Repo)s, %(Build-Depends)s, %(Directory)s, %(Copyright)s, %(Debchangelog)s, %(Changelog)s) " + else: + sql = sql + "(package, version, arch_id, source, srcversion, maintainer, short_description, description, homepage, installed_size, filename, depends, recommends, suggests, size) " + sql = sql + "VALUES (%(Package)s, %(Version)s, %(Architecture)s, %(Source)s, %(Source-Version)s, %(Maintainer)s, %(Short-Description)s, %(Description)s, %(Homepage)s, %(Installed-Size)s, %(Filename)s, %(Depends)s, %(Recommends)s, %(Suggests)s, %(Size)s) " + sql = sql + "RETURNING id" + + cursor.execute(sql, package_content) + package_id = cursor.fetchone()[0] + + if src: + # insert files (dsc, orig.tar.*, ...) + sql = "INSERT INTO apt_files (files, md5_hash, size, source_id) VALUES (%s, %s, %s, %s)" + files_lines = package_content['Files'].split('\n') + for f in files_lines: + f = f.strip().split(" ") + file_md5_hash = f[0] + file_size = f[1] + file_name = f[2] + cursor.execute(sql, (file_name, file_md5_hash, file_size, package_id)) + else: + # insert contents + sql = "INSERT INTO apt_packagecontents (package_id, filename) VALUES (%s, %s)" + for filename in contents[package_content['Package']]: + cursor.execute(sql, (package_id, filename)) + + sql = "INSERT INTO apt_%slist " % (tbl) + if src: + sql = sql + "(suite_id, source_id) " + else: + sql = sql + "(suite_id, package_id) " + sql = sql + "VALUES (%s, %s)" + + cursor.execute(sql, (suite_id, package_id)) + + if src: + sql = "UPDATE apt_srcsuite SET last_update = to_timestamp(%s::int) WHERE (release_id, component_id) = (%s, %s)" + args_tpl = (mtime, release, component) + else: + sql = "UPDATE apt_suite SET last_update = to_timestamp(%s::int) WHERE (release_id, component_id, architecture_id) = (%s, %s, %s)" + args_tpl = (mtime, release, component, architecture) + cursor.execute(sql, args_tpl) + transaction.commit() + +if __name__ == '__main__': + main(sys.argv) -- 2.20.1